{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "SKIP: You need to set the DTM path",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-68fee6a197cf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'DTM_PATH'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"SKIP: You need to set the DTM path\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: SKIP: You need to set the DTM path"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import os\n",
    "from gensim import corpora, utils\n",
    "from gensim.models.wrappers.dtmmodel import DtmModel\n",
    "import numpy as np\n",
    "\n",
    "# if not os.environ.get('DTM_PATH', None):\n",
    "#     raise ValueError(\"SKIP: You need to set the DTM path\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "mypath = '/home/avikbasu/WORK/Economics_of_Innovation/Removed_duplicates_timewise'\n",
    "onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,28) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,7,20,21,22,26,28,32) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,7,26,27,29,32) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,7,21,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,29) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (20,28) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (21,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (7,26,29) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (7,21,22,26,27,29,32) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,7,29,32) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (6,21,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/home/avikbasu/.virtualenvs/oc/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (4,7,21,31) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "yearwise_docs = {}\n",
    "for file in onlyfiles:\n",
    "    df = pd.read_csv(os.path.join(mypath,file), header=None)\n",
    "    yearwise_docs[file] = list(df[10])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove stopwords\n",
    "import re,nltk\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from gensim import corpora, models\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet, stopwords\n",
    "import gensim\n",
    "import os\n",
    "import csv\n",
    "import matplotlib as plt\n",
    "\n",
    "tag_to_type = {'J': wordnet.ADJ, 'V': wordnet.VERB, 'R': wordnet.ADV}\n",
    "\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "    return tag_to_type.get(treebank_tag[:1], wordnet.NOUN)\n",
    "\n",
    "## from Avik Basu\n",
    "def cleanData(doc):\n",
    "    shortword = re.compile(r'\\W*\\b\\w{1,3}\\b')\n",
    "    nonan = re.compile(r'[^a-zA-Z ]')\n",
    "#     tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    stop = stopwords.words('english') + ['paper','system']\n",
    "    lmtzr = WordNetLemmatizer()\n",
    "\n",
    "    tag_to_type = {'J': wordnet.ADJ, 'V': wordnet.VERB, 'R': wordnet.ADV}\n",
    "    tokens = nltk.word_tokenize(shortword.sub('',nonan.sub('',doc.lower())))\n",
    "    tokens = [token for token in tokens if not token in stop]\n",
    "    #print tokens\n",
    "    tags = nltk.pos_tag(tokens)\n",
    "    #print tags\n",
    "    #raw_input()\n",
    "\n",
    "    finalTokens = []\n",
    "    for word, tag in zip(tokens, tags):\n",
    "        finalTokens.append(lmtzr.lemmatize(word, get_wordnet_pos(tag[1])))\n",
    "    return finalTokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on year 2001.csv\n",
      "Working on year 2012.csv\n",
      "Working on year 2004.csv\n",
      "Working on year 2009.csv\n",
      "Working on year 2013.csv\n",
      "Working on year 2007.csv\n",
      "Working on year 2008.csv\n",
      "Working on year 2006.csv\n",
      "Working on year 2003.csv\n",
      "Working on year 2002.csv\n",
      "Working on year 2010.csv\n",
      "Working on year 2005.csv\n",
      "Working on year 2011.csv\n",
      "Working on year 2014.csv\n"
     ]
    }
   ],
   "source": [
    "clean_yearwise = {}\n",
    "for year in yearwise_docs:\n",
    "    print \"Working on year {}\".format(year)\n",
    "    new_docs = []\n",
    "    for doc in yearwise_docs[year]:\n",
    "        clean_doc = cleanData(doc)\n",
    "        new_docs.append(clean_doc)\n",
    "    clean_yearwise[year] = new_docs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "with open('cleaned_yearwise.pkl','wb') as fp:\n",
    "    pkl.dump(clean_yearwise, fp)\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53872"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
